<<<<<<< HEAD
package com.car.crawler;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import com.car.util.DocUtil;

public abstract class BaseClass {
	static Logger logger = Logger.getLogger(DocUtil.class.getName());
	List<String> allurlSet = new ArrayList<String>();
	List<String> notCrawlurlSet = new ArrayList<String>();
	Map<String, Integer> urlType = new HashMap<String, Integer>();
	
	public byte[] signal = new byte[0];
	int threadCount = 20; 
	int waitThreadCount = 0;
	

	/**
	 * @author WangJie
	 * @category get url
	 * @return String
	 */
	protected synchronized String getUrl() {
		if (notCrawlurlSet.isEmpty())
			return null;
		String tmpAUrl = notCrawlurlSet.get(0);
		notCrawlurlSet.remove(0);
		return tmpAUrl;
	}


	protected synchronized void addOneUrl(String url,int type) {
		notCrawlurlSet.add(url);
		allurlSet.add(url);
		urlType.put(url, type);
	}
	
	protected synchronized void addMoreUrl(List<String> urlList,int type) {
		notCrawlurlSet.addAll(urlList);
		allurlSet.addAll(urlList);
		Iterator<String> it = urlList.iterator();
        while (it.hasNext())
        urlType.put(it.next(),type);
	}
	
	public int getThreadCount() {
		return threadCount;
	}
	
	
	

	public void setThreadCount(int threadCount) {
		this.threadCount = threadCount;
	}
	
	/**
	 * download page
	 * @param url
	 * @return Document
	 * @throws Exception 
	 */
	public Document getDoc(String url)  {
		
		Document doc = null;
		int i = 0;
		while (i < 5) {
			try {
				doc = Jsoup.connect(url).timeout(10000).get();
				logger.info("download success:" + url);
				break;
			} catch (Exception e) {
				if (i > 2) {
					logger.info("download wrong :"+url);
				}
				i++;
				if (i >= 5)
					try {
						throw new Exception(e);
					} catch (Exception e1) {
						// TODO 自动生成的 catch 块
						e1.printStackTrace();
					}
			}
		}
		return doc;
	}
	
	
	
	public abstract void threadRun();
	public abstract void start();
	
	/**
	 * 使线程挂起
	 */
	public void threadWait(){
		synchronized (signal) { 
			try {
				waitThreadCount++;
				logger.info( waitThreadCount + " thread are waiting.");
				signal.wait();
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		}
	}
	
	/**
	 * 唤醒线程
	 */
	public void threadNotify(){
		if (waitThreadCount > 0) {
			synchronized (signal) { 
				waitThreadCount--;
				signal.notify();
				logger.warn("唤醒线程:"+signal);
			}
		}
	}
	
	

}
=======
package com.car.crawler;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import com.car.util.DocUtil;

public abstract class BaseClass {
	static Logger logger = Logger.getLogger(DocUtil.class.getName());
	List<String> allurlSet = new ArrayList<String>();
	List<String> notCrawlurlSet = new ArrayList<String>();
	Map<String, Integer> urlType = new HashMap<String, Integer>();
	
	public byte[] signal = new byte[0];
	int threadCount = 20; 
	int waitThreadCount = 0;
	

	/**
	 * @author WangJie
	 * @category get url
	 * @return String
	 */
	protected synchronized String getUrl() {
		if (notCrawlurlSet.isEmpty())
			return null;
		String tmpAUrl = notCrawlurlSet.get(0);
		notCrawlurlSet.remove(0);
		return tmpAUrl;
	}


	protected synchronized void addOneUrl(String url,int type) {
		notCrawlurlSet.add(url);
		allurlSet.add(url);
		urlType.put(url, type);
	}
	
	protected synchronized void addMoreUrl(List<String> urlList,int type) {
		notCrawlurlSet.addAll(urlList);
		allurlSet.addAll(urlList);
		Iterator<String> it = urlList.iterator();
        while (it.hasNext())
        urlType.put(it.next(),type);
	}
	
	public int getThreadCount() {
		return threadCount;
	}
	
	
	

	public void setThreadCount(int threadCount) {
		this.threadCount = threadCount;
	}
	
	/**
	 * download page
	 * @param url
	 * @return Document
	 * @throws Exception 
	 */
	public Document getDoc(String url)  {
		
		Document doc = null;
		int i = 0;
		while (i < 5) {
			try {
				doc = Jsoup.connect(url).timeout(10000).get();
				logger.info("download success:" + url);
				break;
			} catch (Exception e) {
				if (i > 2) {
					logger.info("download wrong :"+url);
				}
				i++;
				if (i >= 5)
					try {
						throw new Exception(e);
					} catch (Exception e1) {
						// TODO 自动生成的 catch 块
						e1.printStackTrace();
					}
			}
		}
		return doc;
	}
	
	
	
	public abstract void threadRun();
	public abstract void start();
	
	/**
	 * 使线程挂起
	 */
	public void threadWait(){
		synchronized (signal) { 
			try {
				waitThreadCount++;
				logger.info( waitThreadCount + " thread are waiting.");
				signal.wait();
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		}
	}
	
	/**
	 * 唤醒线程
	 */
	public void threadNotify(){
		if (waitThreadCount > 0) {
			synchronized (signal) { 
				waitThreadCount--;
				signal.notify();
				logger.warn("唤醒线程:"+signal);
			}
		}
	}
	
	

}
>>>>>>> 12cebf820d4efd66d730623df0b55f6b98421d88
